import pandas as pd
import urllib
import numpy as np
import urllib.request
import re
from textblob import TextBlob
%run lib.py

#name="Legally%20Blonde"
#name="aboutmary"
#name="10Things"
name="magnolia"
#name="Friday%20The%2013th"
#name="Ghost%20Ship"
#name="Juno"
#name="Reservoir+Dogs"
#name="shawshank"
#name="Sixth%20Sense,%20The"
#name="sunset_bld_3_21_49"
#name="Titanic"
#name="toy_story"
#name="trainspotting"
#name="transformers"
#name="the-truman-show_shooting"
#name="batman_production"

ext="html"
txtfiles=["Ghost%20Ship", "Legally%20Blonde", "Friday%20The%2013th", "Juno", "Reservoir+Dogs", "Sixth%20Sense,%20The", "Titanic"]
if name in txtfiles:
    ext="txt"
fp = urllib.request.urlopen("http://www.dailyscript.com/scripts/"+name+"."+ext)
mybytes = fp.read()

mystr = mybytes.decode("utf8", "ignore")
fp.close()
liston=mystr.split("\n")
liston=[s.replace('\r', '') for s in liston]
liston=[re.sub('<[^<]+?>', '', text) for text in liston]

if name=="shawshank":
    liston=[i.replace("\t", "    ") for i in liston]

char=""
script=[]
charintro='                                 '
endofdialogue='          '
dialoguepre='                    '
newscenepre='          '
charintro=''
endofdialogue=''
dialoguepre=''
newscenepre=''
i=45
print("Characters")
i, charintro=nextbigchunk(liston, i)
print("Adverbs")
i, adverb=nextbigchunk(liston, i, adverbs=True)
print("Dialogues")
i, dialoguepre=nextbigchunk(liston, i)
print("New Scene:")
i, newscenepre=nextbigchunk(liston, i)

if newscenepre=="X":
    i=100
    i, newscenepre=nextbigchunk(liston, i)
    if name=="aboutmary":
        newscenepre=" ".join(["" for i in range(56)])
    if len(newscenepre)==len(charintro):
        newscenepre="X"
    

endofdialogue=newscenepre
    

scene=1
for s in liston:
    if s[0:len(charintro)]==charintro and s[len(charintro)]!=" " and s.strip()[0]!="(" and s.strip()[len(s.strip())-1]!=")":
        #print("Charatcer*****")
        char=s[len(charintro):]
        new=dict()
        new['char']=char.strip()
        new['dialogue']=""
        new['scene']=scene
        new['adverb']=""
    if s==endofdialogue or s.replace(" ", "")=="":
        if char!="":
            char=""
            script.append(new)
    if char!="" and s[0:len(dialoguepre)]==dialoguepre and s[len(dialoguepre)]!=" ":
        #print("Dialogue******")
        if new['dialogue']!="":
            new['dialogue']=new['dialogue']+" "
        new['dialogue']=new['dialogue']+s[len(dialoguepre):]
    if char!="" and ((s[0:len(adverb)]==adverb and s[len(adverb)]!=" ") or (len(s)>1 and s.strip()[0]=="(" and s.strip()[len(s.strip())-1]==")" )):
        if new['adverb']!="":
            new['adverb']=new['adverb']+" "
        new['adverb']=new['adverb']+s[len(adverb):]
    if s[0:len(newscenepre)]==newscenepre and len(s)>len(newscenepre) and ( s.isupper()) and s[len(newscenepre)]!=" ":
        scene=scene+1

Characters
                                magnolia
                                NARRATOR
                                NARRATOR
                                NARRATOR
                                NARRATOR
                                NARRATOR
Adverbs
Dialogues
                      In the New York Herald, November 26,
                      year 1911, there is an account of the
                      hanging of three men --
                      ...they died for the murder of
                      Sir Edmund William Godfrey --
                      -- Husband, Father, Pharmacist and all
New Scene:
     a P.T. Anderson picture                             11/10/98
     a Joanne Sellar/Ghoulardi Film Company production

pd.DataFrame(script).to_csv(name+'.csv', index=None)
pd.DataFrame(script)

magnolia=pd.read_csv(name+'.csv')
stopwords = getstopwords()

removedchars=["'S VOICE", "'S WHISPER VOICE", " GATOR"]
for s in removedchars:
    magnolia['char']=magnolia['char'].apply(lambda x: x.replace(s, ""))
i=0
scenes=dict()
for s in magnolia.iterrows():
    scenes[s[1]['scene']]=[]
for s in magnolia.iterrows():
    scenes[s[1]['scene']].append(s[1]['char'])
for s in magnolia.iterrows():
    scenes[s[1]['scene']]=list(set(scenes[s[1]['scene']]))

characters=[]
for s in scenes:
    for k in scenes[s]:
        characters.append(k)
characters=list(set(characters))
appearances=dict()
for s in characters:
    appearances[s]=0
for s in magnolia.iterrows():
    appearances[s[1]['char']]=appearances[s[1]['char']]+1

a=pd.DataFrame(appearances, index=[i for i in range(len(appearances))])

finalcharacters=[]
for s in pd.DataFrame(a.transpose()[0].sort_values(0, ascending=False))[0:10].iterrows():
    finalcharacters.append(s[0])

finalcharacters
file=open(name+"_nodes.csv", "w")
couplesappearances=dict()
for s in finalcharacters:
    file.write(";")
    file.write(s)
file.write("\n")
for s in finalcharacters:
    newlist=[]
    for f in finalcharacters:
        newlist.append(0)
        couplesappearances[f+"_"+s]=0
    j=0
    for f in finalcharacters:
        for p in scenes:
            if f in scenes[p] and s in scenes[p] and f!=s and finalcharacters.index(f)<finalcharacters.index(s): 
                long=len(magnolia[magnolia["scene"]==p])
                newlist[j]=newlist[j]+long
                couplesappearances[f+"_"+s]=couplesappearances[f+"_"+s]+long
        j=j+1
    file.write(s)
    for f in newlist:
        file.write(";")
        file.write(str(f))
    file.write("\n")
file.close()

a=pd.DataFrame(couplesappearances, index=[i for i in range(len(couplesappearances))])
finalcouples=[]
for s in pd.DataFrame(a.transpose()[0].sort_values(0, ascending=False))[0:4].iterrows():
    finalcouples.append(s[0])

file=open(name+"_finalcharacters.csv", "w")
for s in finalcharacters:
    file.write(s+"\n")
file.close()
file=open(name+"_finalcouples.csv", "w")
for s in finalcouples:
    file.write(s+"\n")
file.close()

importantchars=[]
for char in appearances:
    if appearances[char]>10:
        importantchars.append(char)

file=open(name+"_sentiment_overtime_individual.csv", "w")
file2=open(name+"_sentiment_overtime_individualminsmaxs.csv", "w")

for k in finalcharacters:
    print(k)
    dd=getdialogue(magnolia, k, k, scenes)
    dd=[str(d) for d in dd]
    polarities, subjectivities=getsentiment(dd)
    %matplotlib inline
    import matplotlib.pyplot as plt
    moveda=maverage(polarities, dd, .99)
    plt.plot(moveda)
    i=0
    for s in moveda:
        file.write(k+","+str(float(i)/len(moveda))+", "+str(s)+"\n")
        i=i+1
    plt.ylabel('polarities')
    plt.show()
    file2.write(k+"| MIN| "+dd[moveda.index(np.min(moveda))]+"\n")
    file2.write(k+"| MAX| "+dd[moveda.index(np.max(moveda))]+"\n")
    print("MIN: "+dd[moveda.index(np.min(moveda))])
    print("\n")
    print("MAX: "+dd[moveda.index(np.max(moveda))])
    
file.close()
file2.close()

file=open(name+"_sentiment_overtime_couples.csv", "w")
file2=open(name+"_sentiment_overtime_couplesminsmaxs.csv", "w")

for k in finalcouples:
    print(k)
    liston=k.split("_")
    dd=getdialogue(magnolia, liston[0], liston[1], scenes)
    dd=[str(d) for d in dd]
    polarities, subjectivities=getsentiment(dd)
    %matplotlib inline
    import matplotlib.pyplot as plt
    moveda=maverage(polarities, dd, .99)
    plt.plot(moveda)
    i=0
    for s in moveda:
        file.write(k+","+str(float(i)/len(moveda))+", "+str(s)+"\n")
        i=i+1
    plt.ylabel('polarities')
    plt.show()
    file2.write(k+"| MIN| "+dd[moveda.index(np.min(moveda))]+"\n")
    file2.write(k+"| MAX| "+dd[moveda.index(np.max(moveda))]+"\n")
    print("MIN: "+dd[moveda.index(np.min(moveda))])
    print("\n")
    print("MAX: "+dd[moveda.index(np.max(moveda))])
    
file.close()
file2.close()

JIM KURRING

MIN: You mind if I check things back here? 


MAX: YOU'RE OK...you're gonna be ok....
JIMMY

MIN: She went crazy.  She went crazy, Rose. 


MAX: Imagine you are attending a jam session of classical composers and they have  each done an arrangment of the classic  favorite, "Whispering."  Here are three  variations on the theme, as three classic  composer's might have written it -- you are to name the composer.  The First: 
CLAUDIA

MIN: I'm sorry. 


MAX: Did you ever go out with someone and just....lie....question after question, maybe you're trying to  make yourself look cool or better  than you are or whatever, or smarter  or cooler and you just -- not really lie, but maybe you just don't say everything --
FRANK

MIN: If you feel, made to feel like you need them, like -- like you can't live if you're without them or you need, what?  They're pussy?  They're love? Fuck that.  Self Sufficient, gents.  That's the truth. What you are -- we are -- you need them  for what?  To fucking make you a piece of snot rag?  A puppett?  huh?  Hear them bitch and moan? bitch and moan --  and we're taught one thing -- go the other way -- there is No Excuse I will give you, I'm not gonna apologize -- I'm not gonna  apologize for my NEED my DESIRE...my, the  things that I need as a man to feel comfortable... You understand?  You understand?  You need to say something, "my mommy hit me or  daddy hit me or didn't let me play soccer,  so now I make mistakes, cause a that -- something, so now I piss and shit on it and do this." Bullshit.  I'm sorry. ok. yeah. no. fuck.  go.  fuck. alright. go make a new mistake. maybe not, I dunno...fuck.... 


MAX: I wouldn't want that to be misunderstood: My enrollment was totally unoffical because I was, sadly, unable to afford tuition up  there.  But there were three wonderful men who were kind enough to let me sit in on their classes, and they're names are:  Macready, Horn and Langtree among others. I was completely independent financially, and like I said: One Sad Sack A Shit.  So what we're looking at here is a true rags to riches story and I think that's  what most people respond to in "Seduce," And At The End Of The Day? Hey -- it may not  even be about picking up chicks and sticking your cock in it -- it's about finding What You Can Be In This World.  Defining It.  Controling It and  saying: I will take what is mine.  You just happen  to get a blow job out of it, then hey-what-the-fuck- why-not?  he.he.he.
PHIL

MIN: You wanna call him on the phone? We can call him, I can dial the  phone if you can remember the number -- 


MAX: Thank you, Chad, and good luck to you and your mother -- 
STANLEY

MIN: I think that you have to be nicer to me.


MAX: I'm fine. I'm fine, I just wanna keep playing --
DONNIE

MIN: My teeff...my teeef....


MAX: My name is Donnie Smith and I have lot's of love to give. 
EARL

MIN: No, no, the grade...the grade that you're in? 


MAX: "...it's not going to stop 'till you wise up..."
LINDA

MIN: listen...listen to me now, Phil:  I'm sorry, sorry I slapped your face.  ...because I don't know what I'm doing... ...I don't know how to do this, y'know?  You understand?  y'know?  I...I'm...I do things  and I fuck up and I fucked up....forgive me, ok? Can you just...


MAX: I'm listening.  I'm getting better. 
NARRATOR

for key, val in scenes.items():
    for s in scenes[key]:
        new="INSCENE_"+scenes[key][0]
        scenes[key].remove(scenes[key][0])
        scenes[key].append(new)

magnolia.dropna(subset=['dialogue'])
1

1

baskets=[]
spchars=["\"", "'", ".", ",", "-"]
attributes=["?", "!"]
for s in magnolia.iterrows():
    if type(s[1]['dialogue'])!=float and  len(s[1]['dialogue'])>0:
        new=[]
        for k in scenes[s[1]['scene']]:
            new.append(k)
        new.append("SPEAKING_"+s[1]['char'])
        for k in s[1]['dialogue'].split(" "):
            ko=k
            for t in spchars:
                ko=ko.replace(t, "")
            for t in attributes:
                if ko.find(t)>=0:
                    new.append(t)
                    ko=ko.replace(t, "")
            if len(ko)>0:
                new.append(ko.lower())
        new=list(set(new))
        baskets.append(new)

baskets2=[]
basketslist=[]
for k in baskets:
    new=dict()
    new2=[]
    for t in k:
        if t not in stopwords:
            new[t]=1
            new2.append(t)
    baskets2.append(new)
    basketslist.append(new2)

baskets2=pd.DataFrame(baskets2)
from mlxtend.frequent_patterns import apriori
from mlxtend.frequent_patterns import association_rules
baskets2=baskets2.fillna(0)
baskets2.to_csv(name+'_basket.csv')

frequent_itemsets = apriori(baskets2, min_support=5/len(baskets2), use_colnames=True)
rules = association_rules(frequent_itemsets, metric="lift", min_threshold=1)

rules['one_lower']=[int(alllower(i) or alllower(j)) for i, j in zip(rules['antecedants'], rules['consequents'])]

rules['both_lower']=[int(alllower(i) and alllower(j)) for i, j in zip(rules['antecedants'], rules['consequents'])]

rules.to_csv(name+'_rules.csv', index=None)

	adverb	char	dialogue	scene
0		magnolia		1
1		NARRATOR	In the New York Herald, November 26, year 1911...	2
2		NARRATOR	...they died for the murder of Sir Edmund Will...	2
3		NARRATOR	-- Husband, Father, Pharmacist and all around ...	2
4		NARRATOR	Greenberry Hill, London. Population as listed.	3
5		NARRATOR	He was murdered by three vagrants whose motive...	5
6		NARRATOR	...Joseph Green.....	5
7		NARRATOR	...Stanley Berry....	5
8		NARRATOR	...and Nigel Hill...	5
9		NARRATOR	Green, Berry and Hill.	7
10		NARRATOR	...And I Would Like To Think This Was Only A M...	7
11		NARRATOR	As reported in the Reno Gazzette, June of 1983...	9
12		NARRATOR	--- the water that it took to contain the fire --	10
13		NARRATOR	-- and a scuba diver named Delmer Darion.	12
14		NARRATOR	Employee of the Peppermill Hotel and Casino, R...	15
15		NARRATOR	-- well liked and well regarded as a physical,...	16
16		NARRATOR	-- as reported by the coroner, Delmer died of ...	21
17		NARRATOR	...volunteer firefighter, estranged father of ...	24
18		NARRATOR	-- added to this, Mr. Hansen's tortured life m...	26
19		CRAIG HANSEN	...oh God...fuck...I'm sorry...I'm sorry...	27
20		NARRATOR	The weight of the guilt and the measure of coi...	27
21		CRAIG HANSEN	...forgive me...	27
22		NARRATOR	And I Am Trying To Think This Was All Only A M...	29
23		NARRATOR	The tale told at a 1961 awards dinner for the ...	32
24		NARRATOR	Seventeen year old Sydney Barringer. In the ci...	33
25		NARRATOR	The coroner ruled that the unsuccessful suicid...	33
26		NARRATOR	The suicide was confirmed by a note, left in t...	34
27		NARRATOR	At the same time young Sydney stood on the le...	35
28		NARRATOR	The neighbors heard, as they usually did, the...	36
29		NARRATOR	-- and it was not uncommon for them to threat...	37
...	...	...	...	...
1493		DIXON	We gotta get his money so we can get outta her...	382
1494		WORM	That idea is over now. We're not gonna do tha...	382
1495	(to Stanley)	DIXON	DADDY, FUCK, DADDY, DON'T GET MAD AT ME. DON'T...	382
1496		WORM	I'm not mad, son, I will not be mad at you an...	382
1497		DIXON	DAD.	382
1498		DIXON	I - just - thought - that - I - didn't want - ...	382
1499		WORM	It's ok, boy.	382
1500		MUSIC/KERMIT THE FROG	"It's not that easy bein' green... Having to s...	383
1501		DONNIE	My teeff...my teeef....	385
1502		JIM KURRING	YOU'RE OK...you're gonna be ok....	385
1503		NARRATOR	And there is the account of the hanging of thr...	390
1504		NARRATOR	There are stories of coincidence and chance an...	391
1505		NARRATOR	...and we generally say, "Well if that was in...	392
1506		DOCTOR	Are you with us? Linda? Is it Linda?	394
1507		NARRATOR	Someone's so and so meet someone else's so and...	395
1508		NARRATOR	And it is in the humble opinion of this narrat...	398
1509		STANLEY	Dad...Dad.	399
1510		STANLEY	You have to be nicer to me, Dad.	399
1511		RICK	Go to bed.	399
1512		STANLEY	I think that you have to be nicer to me.	399
1513		RICK	Go to bed.	399
1514		NARRATOR	...and so it goes and so it goes and the book...	400
1515		MARCIE	I killed him. I killed my husband. He hit my...	401
1516		DONNIE	I know that I did a thtupid thing. Tho-thtupid...	402
1517		DONNIE	I really do hath love to give, I juth don't kn...	402
1518		JIM KURRING	...these security systems can be a real joke. ...	403
1519		DONNIE	....ohh-thur-I-thur-thill....	403
1520		JIM KURRING	You guys make alotta money, huh?	403
1521	(beat)	JIM KURRING	...alot of people think this is just a job tha...	405
1522		END.		406

sentiment	Porcentaje
positive	56.2%
negative	43.8%

sentiment	Porcentaje
positive	20.7%
negative	15.0%
anticipation	12.3%
trust	12.3%
joy	10.5%
sadness	6.8%
fear	6.6%
surprise	5.5%
anger	5.3%
disgust	5.0%

sentiment	Porcentaje
negative	37.8%
positive	30.1%
uncertainty	25.7%
litigious	4.7%
constraining	1.7%

sentiment	Porcentaje
positive	52.98%
negative	47.02%

Analisis de Script Pelicula Titanic

Bernardo Garcia & Eduardo Hidalgo

May 9, 2018

Agenda

Procesamiento de Texto

Ver archivo Python

Analisis de Sentimiento (Pelicula & Personaje)

Score por Pelicula

Score por Personaje

Score por Personaje en el tiempo

Top 10 Personajes

Top 4 Parejas

Reglas de Asociación entre palabras (Market Basket)

Toda la pelicula

Top 10 Personajes

Top 4 Parejas

Analisis de Relaciones entre Personajes (Pagerank)

sentiment	Porcentaje
positive	19.9%
negative	17.2%
trust	12.1%
anticipation	11.6%
joy	10.1%
sadness	8.3%
fear	6.7%
anger	4.9%
surprise	4.7%
disgust	4.5%

sentiment	Porcentaje
negative	44.1%
uncertainty	32.2%
positive	18.6%
litigious	3.4%
constraining	1.7%

sentiment	Porcentaje
positive	17.3%
negative	15.1%
anticipation	12.0%
trust	12.0%
joy	10.9%
fear	8.7%
anger	7.3%
sadness	6.7%
disgust	5.0%
surprise	5.0%

sentiment	Porcentaje
positive	21.7%
negative	14.0%
joy	12.7%
trust	12.3%
anticipation	12.0%
disgust	7.0%
anger	5.3%
sadness	5.3%
surprise	5.0%
fear	4.7%

sentiment	Porcentaje
positive	37.2%
uncertainty	34.9%
negative	20.9%
litigious	4.7%
constraining	2.3%

sentiment	Porcentaje
positive	26.3%
joy	17.0%
anticipation	14.6%
trust	13.5%
negative	9.4%
sadness	6.4%
fear	5.3%
anger	2.9%
surprise	2.9%
disgust	1.8%

sentiment	Porcentaje
positive	16.7%
negative	15.4%
anticipation	14.1%
trust	12.8%
joy	9.0%
fear	7.7%
sadness	7.7%
disgust	6.4%
surprise	6.4%
anger	3.8%

sentiment	Porcentaje
positive	35.7%
negative	28.6%
uncertainty	21.4%
constraining	7.1%
litigious	7.1%

sentiment	Porcentaje
negative	26.7%
positive	18.7%
anticipation	8.0%
disgust	8.0%
sadness	8.0%
trust	8.0%
fear	6.7%
joy	6.7%
anger	5.3%
surprise	4.0%

sentiment	Porcentaje
positive	20.8%
negative	15.1%
trust	13.2%
anger	11.3%
anticipation	11.3%
surprise	9.4%
disgust	7.5%
joy	7.5%
fear	1.9%
sadness	1.9%

sentiment	Porcentaje
anticipation	22.5%
positive	17.5%
negative	12.5%
trust	12.5%
fear	10.0%
joy	7.5%
sadness	7.5%
surprise	5.0%
anger	2.5%
disgust	2.5%

sentiment	Porcentaje
positive	23.1%
anticipation	17.9%
trust	15.4%
joy	11.5%
negative	10.3%
surprise	9.0%
fear	3.8%
sadness	3.8%
anger	2.6%
disgust	2.6%

sentiment	Porcentaje
positive	25.0%
trust	21.4%
negative	14.3%
sadness	14.3%
anticipation	10.7%
disgust	3.6%
fear	3.6%
joy	3.6%
surprise	3.6%

sentiment	Porcentaje
positive	30.8%
anticipation	23.1%
negative	15.4%
trust	15.4%
fear	7.7%
joy	7.7%

sentiment	Porcentaje
positive	24.3%
anticipation	16.2%
trust	16.2%
joy	10.8%
negative	10.8%
fear	8.1%
sadness	5.4%
anger	2.7%
disgust	2.7%
surprise	2.7%

Personaje	Min_Max	Dialogo
ROSE	MIN	Jack, this is impossible. I can’t see you.
ROSE	MAX	Titanic was called the Ship of Dreams. And it was. It really was…
JACK	MIN	I just seem to spew ’em out. Besides, they’re not worth a damn anyway.
JACK	MAX	It brought me to you. And I’m thankful, Rose. I’m thankful.
CAL	MIN	God damn it to hell! Come on.
CAL	MAX	Probably best. It’ll be all business and politics, that sort of thing. Wouldn’t interest you. Good of you to come.
LOVETT	MIN	Would you like anything?
LOVETT	MAX	Just let me hold it in my hand, Rose. Please. Just once.
RUTH	MIN	Will the lifeboats be seated according to class? I hope they’re not too crowded –
RUTH	MAX	Rose, this is not a game! Our situation is precarious. You know the money’s gone!
MOLLY	MIN	Come on, you heard the man. Get in the boat, sister.
MOLLY	MAX	Well, Jack, it sounds like you’re a good man to have around in a sticky spot –
BODINE	MIN	Ooohh daddy-oh, are you seein’ what I’m seein’?
BODINE	MAX	We’ve put together the world’s largest database on the Titanic. Okay, here…
SMITH	MIN	Can you shore up?
SMITH	MAX	No, but we’re making excellent time.
ANDREWS	MIN	Anyone in here?
ANDREWS	MAX	Sleep soundly, young Rose. I have built you a good ship, strong and true. She’s all the lifeboat you need.
LOVEJOY	MIN	I’ve found her. She’s just over on the port side. With him.
LOVEJOY	MAX	Miss Rose? Hello?

Parejas	Min_Max	Dialogo
ROSE_JACK	MIN	And these were drawn from life?
ROSE_JACK	MAX	Right.
ROSE_CAL	MIN	I’m not going without you.
ROSE_CAL	MAX	There’s the Countess Rothes. And that’s John Jacob Astor… the richest man on the ship. His little wifey there, Madeleine, is my age and in a delicate condition. See how she’s trying to hide it. Quite the scandal. And over there, that’s Sir Cosmo and Lucile, Lady Duff-Gordon. She designs naughty lingerie, among her many talents. Very popular with the royals.
ROSE_RUTH	MIN	You unimaginable bastard.
ROSE_RUTH	MAX	I don’t understand you. It is a fine match with Hockley, and it will insure our survival.
CAL_RUTH	MIN	Will the lifeboats be seated according to class? I hope they’re not too crowded –
CAL_RUTH	MAX	A real man makes his own luck, Archie.

Numero de Dialogos	Lift Minimo	Lift Maximo
50,670	-3	3
101,482	3	10
138,260	10	16
112,646	16	23
112,166	23	30
52,868	30	36

Numero de Dialogos	Leverage Minimo	Leverage Maximo
10,778	-0.0018	0.0018
730,768	0.0018	0.0055
348,438	0.0055	0.0091
17,900	0.0091	0.013
15,232	0.013	0.016
7,210	0.016	0.02